<!DOCTYPE html><html lang="zh-CN" data-theme="light"><head><meta charset="UTF-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width,initial-scale=1"><title>全栈进阶那些事 - 一只爱技术的程序猿，想把分享变成一种习惯！ </title><meta name="author" content="liuyuantao"><meta name="copyright" content="liuyuantao"><meta name="format-detection" content="telephone=no"><meta name="theme-color" content="#ffffff"><meta name="description" content="一个有趣、有料、有内涵的地方!">
<meta property="og:type" content="website">
<meta property="og:title" content="全栈进阶那些事">
<meta property="og:url" content="https://liuyuantao.gitee.io/page/3/index.html">
<meta property="og:site_name" content="全栈进阶那些事">
<meta property="og:description" content="一个有趣、有料、有内涵的地方!">
<meta property="og:locale" content="zh_CN">
<meta property="og:image" content="https://liuyuantao.gitee.io/img/zhiyao_avatar.jpg">
<meta property="article:author" content="liuyuantao">
<meta name="twitter:card" content="summary">
<meta name="twitter:image" content="https://liuyuantao.gitee.io/img/zhiyao_avatar.jpg"><link rel="shortcut icon" href="/img/favicon.png"><link rel="canonical" href="https://liuyuantao.gitee.io/page/3/"><link rel="preconnect" href="//cdn.jsdelivr.net"/><link rel="preconnect" href="//hm.baidu.com"/><link rel="preconnect" href="//busuanzi.ibruce.info"/><link rel="stylesheet" href="/css/index.css"><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@fortawesome/fontawesome-free/css/all.min.css" media="print" onload="this.media='all'"><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/node-snackbar/dist/snackbar.min.css" media="print" onload="this.media='all'"><script>var _hmt = _hmt || [];
(function() {
  var hm = document.createElement("script");
  hm.src = "https://hm.baidu.com/hm.js?d43383137144f76a37f1074576bc674e";
  var s = document.getElementsByTagName("script")[0]; 
  s.parentNode.insertBefore(hm, s);
})();
</script><script>const GLOBAL_CONFIG = { 
  root: '/',
  algolia: undefined,
  localSearch: {"path":"search.xml","languages":{"hits_empty":"找不到您查询的内容：${query}"}},
  translate: {"defaultEncoding":1,"translateDelay":0,"msgToTraditionalChinese":"繁","msgToSimplifiedChinese":"简"},
  noticeOutdate: undefined,
  highlight: {"plugin":"highlighjs","highlightCopy":true,"highlightLang":true,"highlightHeightLimit":200},
  copy: {
    success: '复制成功',
    error: '复制错误',
    noSupport: '浏览器不支持'
  },
  relativeDate: {
    homepage: true,
    post: true
  },
  runtime: '天',
  date_suffix: {
    just: '刚刚',
    min: '分钟前',
    hour: '小时前',
    day: '天前',
    month: '个月前'
  },
  copyright: {"limitCount":500,"languages":{"author":"作者: liuyuantao","link":"链接: ","source":"来源: 全栈进阶那些事","info":"著作权归作者所有。商业转载请联系作者获得授权，非商业转载请注明出处。"}},
  lightbox: 'mediumZoom',
  Snackbar: {"chs_to_cht":"你已切换为繁体","cht_to_chs":"你已切换为简体","day_to_night":"你已切换为深色模式","night_to_day":"你已切换为浅色模式","bgLight":"#49b1f5","bgDark":"#2d3035","position":"top-right"},
  source: {
    jQuery: 'https://cdn.jsdelivr.net/npm/jquery@latest/dist/jquery.min.js',
    justifiedGallery: {
      js: 'https://cdn.jsdelivr.net/npm/justifiedGallery/dist/js/jquery.justifiedGallery.min.js',
      css: 'https://cdn.jsdelivr.net/npm/justifiedGallery/dist/css/justifiedGallery.min.css'
    },
    fancybox: {
      js: 'https://cdn.jsdelivr.net/npm/@fancyapps/fancybox@latest/dist/jquery.fancybox.min.js',
      css: 'https://cdn.jsdelivr.net/npm/@fancyapps/fancybox@latest/dist/jquery.fancybox.min.css'
    }
  },
  isPhotoFigcaption: true,
  islazyload: false,
  isanchor: true
}</script><script id="config-diff">var GLOBAL_CONFIG_SITE = {
  title: '全栈进阶那些事',
  isPost: false,
  isHome: true,
  isHighlightShrink: false,
  isToc: false,
  postUpdate: '2021-11-23 18:55:57'
}</script><noscript><style type="text/css">
  #nav {
    opacity: 1
  }
  .justified-gallery img {
    opacity: 1
  }

  #recent-posts time,
  #post-meta time {
    display: inline !important
  }
</style></noscript><script>(win=>{
    win.saveToLocal = {
      set: function setWithExpiry(key, value, ttl) {
        if (ttl === 0) return
        const now = new Date()
        const expiryDay = ttl * 86400000
        const item = {
          value: value,
          expiry: now.getTime() + expiryDay,
        }
        localStorage.setItem(key, JSON.stringify(item))
      },

      get: function getWithExpiry(key) {
        const itemStr = localStorage.getItem(key)

        if (!itemStr) {
          return undefined
        }
        const item = JSON.parse(itemStr)
        const now = new Date()

        if (now.getTime() > item.expiry) {
          localStorage.removeItem(key)
          return undefined
        }
        return item.value
      }
    }
  
    win.getScript = url => new Promise((resolve, reject) => {
      const script = document.createElement('script')
      script.src = url
      script.async = true
      script.onerror = reject
      script.onload = script.onreadystatechange = function() {
        const loadState = this.readyState
        if (loadState && loadState !== 'loaded' && loadState !== 'complete') return
        script.onload = script.onreadystatechange = null
        resolve()
      }
      document.head.appendChild(script)
    })
  
      win.activateDarkMode = function () {
        document.documentElement.setAttribute('data-theme', 'dark')
        if (document.querySelector('meta[name="theme-color"]') !== null) {
          document.querySelector('meta[name="theme-color"]').setAttribute('content', '#0d0d0d')
        }
      }
      win.activateLightMode = function () {
        document.documentElement.setAttribute('data-theme', 'light')
        if (document.querySelector('meta[name="theme-color"]') !== null) {
          document.querySelector('meta[name="theme-color"]').setAttribute('content', '#ffffff')
        }
      }
      const t = saveToLocal.get('theme')
    
          if (t === 'dark') activateDarkMode()
          else if (t === 'light') activateLightMode()
        
      const asideStatus = saveToLocal.get('aside-status')
      if (asideStatus !== undefined) {
        if (asideStatus === 'hide') {
          document.documentElement.classList.add('hide-aside')
        } else {
          document.documentElement.classList.remove('hide-aside')
        }
      }
    
    const fontSizeVal = saveToLocal.get('global-font-size')
    if (fontSizeVal !== undefined) {
      document.documentElement.style.setProperty('--global-font-size', fontSizeVal + 'px')
    }
    
    const detectApple = () => {
      if (GLOBAL_CONFIG_SITE.isHome && /iPad|iPhone|iPod|Macintosh/.test(navigator.userAgent)){
        document.documentElement.classList.add('apple')
      }
    }
    detectApple()
    })(window)</script><meta name="generator" content="Hexo 5.4.0"><link rel="alternate" href="/atom.xml" title="全栈进阶那些事" type="application/atom+xml">
</head><body><div id="loading-box"><div class="loading-left-bg"></div><div class="loading-right-bg"></div><div class="spinner-box"><div class="configure-border-1"><div class="configure-core"></div></div><div class="configure-border-2"><div class="configure-core"></div></div><div class="loading-word">加载中...</div></div></div><div id="sidebar"><div id="menu-mask"></div><div id="sidebar-menus"><div class="avatar-img is-center"><img src="/img/zhiyao_avatar.jpg" onerror="onerror=null;src='/img/friend_404.gif'" alt="avatar"/></div><div class="site-data"><div class="data-item is-center"><div class="data-item-link"><a href="/archives/"><div class="headline">文章</div><div class="length-num">41</div></a></div></div><div class="data-item is-center"><div class="data-item-link"><a href="/tags/"><div class="headline">标签</div><div class="length-num">14</div></a></div></div><div class="data-item is-center"><div class="data-item-link"><a href="/categories/"><div class="headline">分类</div><div class="length-num">4</div></a></div></div></div><hr/><div class="menus_items"><div class="menus_item"><a class="site-page" href="/"><i class="fa-fw fas fa-home"></i><span> 首页</span></a></div><div class="menus_item"><a class="site-page" href="/archives/"><i class="fa-fw fas fa-archive"></i><span> 时间轴</span></a></div><div class="menus_item"><a class="site-page" href="/tags/"><i class="fa-fw fas fa-tags"></i><span> 标签</span></a></div><div class="menus_item"><a class="site-page" href="/categories/"><i class="fa-fw fas fa-folder-open"></i><span> 分类</span></a></div><div class="menus_item"><a class="site-page" href="/link/"><i class="fa-fw fas fa-link"></i><span> 友链</span></a></div><div class="menus_item"><a class="site-page" href="/about/"><i class="fa-fw fas fa-heart"></i><span> 关于</span></a></div></div></div></div><div class="page" id="body-wrap"><header class="not-top-img" id="page-header"><nav id="nav"><span id="blog_name"><a id="site-name" href="/">全栈进阶那些事</a></span><div id="menus"><div id="search-button"><a class="site-page social-icon search"><i class="fas fa-search fa-fw"></i><span> 搜索</span></a></div><div class="menus_items"><div class="menus_item"><a class="site-page" href="/"><i class="fa-fw fas fa-home"></i><span> 首页</span></a></div><div class="menus_item"><a class="site-page" href="/archives/"><i class="fa-fw fas fa-archive"></i><span> 时间轴</span></a></div><div class="menus_item"><a class="site-page" href="/tags/"><i class="fa-fw fas fa-tags"></i><span> 标签</span></a></div><div class="menus_item"><a class="site-page" href="/categories/"><i class="fa-fw fas fa-folder-open"></i><span> 分类</span></a></div><div class="menus_item"><a class="site-page" href="/link/"><i class="fa-fw fas fa-link"></i><span> 友链</span></a></div><div class="menus_item"><a class="site-page" href="/about/"><i class="fa-fw fas fa-heart"></i><span> 关于</span></a></div></div><div id="toggle-menu"><a class="site-page"><i class="fas fa-bars fa-fw"></i></a></div></div></nav></header><main class="layout" id="content-inner"><div class="recent-posts" id="recent-posts"><div class="recent-post-item"><div class="post_cover left_radius"><a href="/2020/11/07/bc127bac6366.html" title="Hadoop数据压缩使用介绍">     <img class="post_bg" src="https://oscimg.oschina.net/oscnet/up-f1ae1539eaceaa70955891089aa6900f66a.png" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="Hadoop数据压缩使用介绍"></a></div><div class="recent-post-info"><a class="article-title" href="/2020/11/07/bc127bac6366.html" title="Hadoop数据压缩使用介绍">Hadoop数据压缩使用介绍</a><div class="article-meta-wrap"><span class="post-meta-date"><i class="far fa-calendar-alt"></i><span class="article-meta-label">发表于</span><time datetime="2020-11-07T05:27:49.000Z" title="发表于 2020-11-07 13:27:49">2020-11-07</time></span><span class="article-meta"><span class="article-meta__separator">|</span><i class="fas fa-inbox"></i><a class="article-meta__categories" href="/categories/%E5%A4%A7%E6%95%B0%E6%8D%AE/">大数据</a></span><span class="article-meta tags"><span class="article-meta__separator">|</span><i class="fas fa-tag"></i><a class="article-meta__tags" href="/tags/Hadoop/">Hadoop</a></span></div><div class="content">一、压缩原则（1）运算密集型的Job，少用压缩
（2）IO密集型的Job，多用压缩
二、压缩算法比较
三、压缩位置选择

四、压缩参数配置
1）为了支持多种压缩/解压缩算法，Hadoop引入了编码/解码器

2）要在Hadoop中启用压缩，可以配置如下参数

</div></div></div><div class="recent-post-item"><div class="post_cover right_radius"><a href="/2020/11/07/47c26d75ab86.html" title="NameNode和SecondaryNameNode的工作机制">     <img class="post_bg" src="https://oscimg.oschina.net/oscnet/up-e9146688167ee453fc810bd2ad75aa10659.png" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="NameNode和SecondaryNameNode的工作机制"></a></div><div class="recent-post-info"><a class="article-title" href="/2020/11/07/47c26d75ab86.html" title="NameNode和SecondaryNameNode的工作机制">NameNode和SecondaryNameNode的工作机制</a><div class="article-meta-wrap"><span class="post-meta-date"><i class="far fa-calendar-alt"></i><span class="article-meta-label">发表于</span><time datetime="2020-11-07T05:22:57.000Z" title="发表于 2020-11-07 13:22:57">2020-11-07</time></span><span class="article-meta"><span class="article-meta__separator">|</span><i class="fas fa-inbox"></i><a class="article-meta__categories" href="/categories/%E5%A4%A7%E6%95%B0%E6%8D%AE/">大数据</a></span><span class="article-meta tags"><span class="article-meta__separator">|</span><i class="fas fa-tag"></i><a class="article-meta__tags" href="/tags/Hadoop/">Hadoop</a><span class="article-meta__link">•</span><a class="article-meta__tags" href="/tags/HDFS/">HDFS</a></span></div><div class="content">
1）第一阶段：NameNode启动 （1）第一次启动NameNode格式化后，创建Fsimage和Edits文件。如果不是第一次启动，直接加载编辑日志和镜像文件到内存。
（2）客户端对元数据进行增删改的请求。
（3）NameNode记录操作日志，更新滚动日志。
（4）NameNode在内存中对元数据进行增删改。
2）第二阶段：Secondary NameNode工作（1）Secondary NameNode询问NameNode是否需要CheckPoint。直接带回NameNode是否检查结果。
（2）Secondary NameNode请求执行CheckPoint。
（3）NameNode滚动正在写的Edits日志。
（4）将滚动前的编辑日志和镜像文件拷贝到Secondary NameNode。
（5）Secondary NameNode加载编辑日志和镜像文件到内存，并合并。
（6）生成新的镜像文件fsimage.chkpoint。
（7）拷贝fsimage.chkpoint到NameNode。
（8）NameNode将fsimage.chkpoint重新命名成fsimage。
</div></div></div><div class="recent-post-item"><div class="post_cover left_radius"><a href="/2020/11/07/7d14a19c4606.html" title="DataNode工作机制">     <img class="post_bg" src="https://oscimg.oschina.net/oscnet/up-0a831a2854d1859bc355a98a741ee74452b.png" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="DataNode工作机制"></a></div><div class="recent-post-info"><a class="article-title" href="/2020/11/07/7d14a19c4606.html" title="DataNode工作机制">DataNode工作机制</a><div class="article-meta-wrap"><span class="post-meta-date"><i class="far fa-calendar-alt"></i><span class="article-meta-label">发表于</span><time datetime="2020-11-07T05:22:28.000Z" title="发表于 2020-11-07 13:22:28">2020-11-07</time></span><span class="article-meta"><span class="article-meta__separator">|</span><i class="fas fa-inbox"></i><a class="article-meta__categories" href="/categories/%E5%A4%A7%E6%95%B0%E6%8D%AE/">大数据</a></span><span class="article-meta tags"><span class="article-meta__separator">|</span><i class="fas fa-tag"></i><a class="article-meta__tags" href="/tags/Hadoop/">Hadoop</a><span class="article-meta__link">•</span><a class="article-meta__tags" href="/tags/HDFS/">HDFS</a></span></div><div class="content">
（1）一个数据块在DataNode上以文件形式存储在磁盘上，包括两个文件，一个是数据本身，一个是元数据包括数据块的长度，块数据的校验和，以及时间戳。
（2）DataNode启动后向NameNode注册，通过后，周期性（6小时）的向NameNode上报所有的块信息。
DN向NN汇报当前解读信息的时间间隔，默认6小时；
&lt;property&gt;    &lt;name&gt;dfs.blockreport.intervalMsec&lt;/name&gt;    &lt;value&gt;21600000&lt;/value&gt;    &lt;description&gt;Determines block reporting interval in milliseconds.&lt;/description&gt;&lt;/property&gt;

DN扫描自己节点块信息列表的时间，默认6小时
&lt;property&gt;    &lt;name&gt;dfs.datanode.directoryscan.interval&lt;/name&gt;    &lt;val ...</div></div></div><div class="recent-post-item"><div class="post_cover right_radius"><a href="/2020/11/07/76f2b2981271.html" title="NameNode内存生产配置">     <img class="post_bg" src="https://oscimg.oschina.net/oscnet/up-9144464900899783cb2148ae49ec593740d.png" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="NameNode内存生产配置"></a></div><div class="recent-post-info"><a class="article-title" href="/2020/11/07/76f2b2981271.html" title="NameNode内存生产配置">NameNode内存生产配置</a><div class="article-meta-wrap"><span class="post-meta-date"><i class="far fa-calendar-alt"></i><span class="article-meta-label">发表于</span><time datetime="2020-11-07T05:21:47.000Z" title="发表于 2020-11-07 13:21:47">2020-11-07</time></span><span class="article-meta"><span class="article-meta__separator">|</span><i class="fas fa-inbox"></i><a class="article-meta__categories" href="/categories/%E5%A4%A7%E6%95%B0%E6%8D%AE/">大数据</a></span><span class="article-meta tags"><span class="article-meta__separator">|</span><i class="fas fa-tag"></i><a class="article-meta__tags" href="/tags/Hadoop/">Hadoop</a><span class="article-meta__link">•</span><a class="article-meta__tags" href="/tags/HDFS/">HDFS</a></span></div><div class="content">Hadoop2.x系列，配置NameNode内存NameNode内存默认2000m，如果服务器内存4G，NameNode内存可以配置3g。在hadoop-env.sh文件中配置如下。
HADOOP_NAMENODE_OPTS=-Xmx3072m

Hadoop3.x系列，配置NameNode内存经验分享：

namenode最小值1G，每增加100万个block，增加1G内存。datanode最小值4G，block或者副本数升高，都应该调大datanode的值。一个datanode上的副本总数低于400万时调为4G，超过400万后，每增加100万，增加1G。

具体修改：hadoop-env.sh
export HDFS_NAMENODE_OPTS=&quot;-Dhadoop.security.logger=INFO,RFAS -Xmx1024m&quot;export HDFS_DATANODE_OPTS=&quot;-Dhadoop.security.logger=ERROR,RFAS -Xmx1024m&quot;

namenode大约在280行。
 
datanode大约在 ...</div></div></div><div class="recent-post-item"><div class="post_cover left_radius"><a href="/2020/11/07/2e976324e14a.html" title="hadoop开启回收站配置">     <img class="post_bg" src="https://oscimg.oschina.net/oscnet/up-1d5d94516aaffad5582342bb6d060d71fe8.png" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="hadoop开启回收站配置"></a></div><div class="recent-post-info"><a class="article-title" href="/2020/11/07/2e976324e14a.html" title="hadoop开启回收站配置">hadoop开启回收站配置</a><div class="article-meta-wrap"><span class="post-meta-date"><i class="far fa-calendar-alt"></i><span class="article-meta-label">发表于</span><time datetime="2020-11-07T05:20:51.000Z" title="发表于 2020-11-07 13:20:51">2020-11-07</time></span><span class="article-meta"><span class="article-meta__separator">|</span><i class="fas fa-inbox"></i><a class="article-meta__categories" href="/categories/%E5%A4%A7%E6%95%B0%E6%8D%AE/">大数据</a></span><span class="article-meta tags"><span class="article-meta__separator">|</span><i class="fas fa-tag"></i><a class="article-meta__tags" href="/tags/Hadoop/">Hadoop</a><span class="article-meta__link">•</span><a class="article-meta__tags" href="/tags/HDFS/">HDFS</a></span></div><div class="content">开启回收站功能，可以将删除的文件在不超时的情况下，恢复原数据，起到防止误删除、备份等作用。

开启回收站功能参数说明
（1）默认值fs.trash.interval = 0，0表示禁用回收站；其他值表示设置文件的存活时间。
（2）默认值fs.trash.checkpoint.interval = 0，检查回收站的间隔时间。如果该值为0，则该值设置和fs.trash.interval的参数值相等。
（3）要求fs.trash.checkpoint.interval &lt;= fs.trash.interval。
启用回收站
修改core-site.xml，配置垃圾回收时间为30分钟。
回收站目录在HDFS集群中的路径：/user/&#123;username&#125;/.Trash/….
注意：通过网页上直接删除的文件也不会走回收站。
只有在命令行利用hadoop fs -rm命令删除的文件才会走回收站。
</div></div></div><div class="recent-post-item"><div class="post_cover right_radius"><a href="/2020/11/07/2296d5b58502.html" title="Hadoop集群数据均衡之磁盘间数据均衡">     <img class="post_bg" src="https://oscimg.oschina.net/oscnet/up-91c922f361630fb00034d9eeb4d7ccb04e0.png" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="Hadoop集群数据均衡之磁盘间数据均衡"></a></div><div class="recent-post-info"><a class="article-title" href="/2020/11/07/2296d5b58502.html" title="Hadoop集群数据均衡之磁盘间数据均衡">Hadoop集群数据均衡之磁盘间数据均衡</a><div class="article-meta-wrap"><span class="post-meta-date"><i class="far fa-calendar-alt"></i><span class="article-meta-label">发表于</span><time datetime="2020-11-07T05:20:07.000Z" title="发表于 2020-11-07 13:20:07">2020-11-07</time></span><span class="article-meta"><span class="article-meta__separator">|</span><i class="fas fa-inbox"></i><a class="article-meta__categories" href="/categories/%E5%A4%A7%E6%95%B0%E6%8D%AE/">大数据</a></span><span class="article-meta tags"><span class="article-meta__separator">|</span><i class="fas fa-tag"></i><a class="article-meta__tags" href="/tags/Hadoop/">Hadoop</a><span class="article-meta__link">•</span><a class="article-meta__tags" href="/tags/HDFS/">HDFS</a></span></div><div class="content">生产环境，由于硬盘空间不足，往往需要增加一块硬盘。刚加载的硬盘没有数据时，可以执行磁盘数据均衡命令。（Hadoop3.x新特性）

plan后面带的节点的名字必须是已经存在的，并且是需要均衡的节点。
如果节点不存在，会报如下错误：

如果节点只有一个硬盘的话，不会创建均衡计划：

（1）生成均衡计划hdfs diskbalancer -plan hadoop102

（2）执行均衡计划hdfs diskbalancer -execute hadoop102.plan.json

（3）查看当前均衡任务的执行情况hdfs diskbalancer -query hadoop103

（4）取消均衡任务hdfs diskbalancer -cancel hadoop102.plan.json
</div></div></div><div class="recent-post-item"><div class="post_cover left_radius"><a href="/2020/11/07/5f9b1cc664c1.html" title="HDFS—集群扩容及缩容">     <img class="post_bg" src="https://oscimg.oschina.net/oscnet/up-acb08d98e431429592cdb8802409bb29a19.png" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="HDFS—集群扩容及缩容"></a></div><div class="recent-post-info"><a class="article-title" href="/2020/11/07/5f9b1cc664c1.html" title="HDFS—集群扩容及缩容">HDFS—集群扩容及缩容</a><div class="article-meta-wrap"><span class="post-meta-date"><i class="far fa-calendar-alt"></i><span class="article-meta-label">发表于</span><time datetime="2020-11-07T05:19:32.000Z" title="发表于 2020-11-07 13:19:32">2020-11-07</time></span><span class="article-meta"><span class="article-meta__separator">|</span><i class="fas fa-inbox"></i><a class="article-meta__categories" href="/categories/%E5%A4%A7%E6%95%B0%E6%8D%AE/">大数据</a></span><span class="article-meta tags"><span class="article-meta__separator">|</span><i class="fas fa-tag"></i><a class="article-meta__tags" href="/tags/Hadoop/">Hadoop</a><span class="article-meta__link">•</span><a class="article-meta__tags" href="/tags/HDFS/">HDFS</a></span></div><div class="content">白名单：表示在白名单的主机IP地址可以，用来存储数据。

配置白名单步骤如下：
1）在NameNode节点的/opt/module/hadoop-3.1.4/etc/hadoop目录下分别创建whitelist 和blacklist文件
（1）创建白名单
[lytfly@hadoop102 hadoop]$ vim whitelist

在whitelist中添加如下主机名称，假如集群正常工作的节点为102 103
hadoop102hadoop103

（2）创建黑名单
[lytfly@hadoop102 hadoop]$ touch blacklist

保持空的就可以
2）在hdfs-site.xml配置文件中增加dfs.hosts配置参数
&lt;!-- 白名单 --&gt;&lt;property&gt;     &lt;name&gt;dfs.hosts&lt;/name&gt;     &lt;value&gt;/opt/module/hadoop-3.1.4/etc/hadoop/whitelist&lt;/value&gt;&lt;/property&gt;&lt;! ...</div></div></div><div class="recent-post-item"><div class="post_cover right_radius"><a href="/2020/11/07/54ccdf4e0331.html" title="Hadoop服务器间数据均衡">     <img class="post_bg" src="https://oscimg.oschina.net/oscnet/up-9a4fb4e45e0902123918f03c0bff131a3b8.png" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="Hadoop服务器间数据均衡"></a></div><div class="recent-post-info"><a class="article-title" href="/2020/11/07/54ccdf4e0331.html" title="Hadoop服务器间数据均衡">Hadoop服务器间数据均衡</a><div class="article-meta-wrap"><span class="post-meta-date"><i class="far fa-calendar-alt"></i><span class="article-meta-label">发表于</span><time datetime="2020-11-07T05:19:02.000Z" title="发表于 2020-11-07 13:19:02">2020-11-07</time></span><span class="article-meta"><span class="article-meta__separator">|</span><i class="fas fa-inbox"></i><a class="article-meta__categories" href="/categories/%E5%A4%A7%E6%95%B0%E6%8D%AE/">大数据</a></span><span class="article-meta tags"><span class="article-meta__separator">|</span><i class="fas fa-tag"></i><a class="article-meta__tags" href="/tags/Hadoop/">Hadoop</a></span></div><div class="content">在企业开发中，如果经常在hadoop102和hadoop104上提交任务，且副本数为2，由于数据本地性原则，就会导致hadoop102和hadoop104数据过多，hadoop103存储的数据量小。
另一种情况，就是新服役的服务器数据量比较少，需要执行集群均衡命令。

开启数据均衡命令：
[lytfly@hadoop105 hadoop-3.1.4]$ sbin/start-balancer.sh -threshold 10

对于参数10，代表的是集群中各个节点的磁盘空间利用率相差不超过10%，可根据实际情况进行调整。
停止数据均衡命令：
[lytfly@hadoop105 hadoop-3.1.4]$ sbin/stop-balancer.sh

注意：由于HDFS需要启动单独的Rebalance Server来执行Rebalance操作，所以尽量不要在NameNode上执行start-balancer.sh，而是找一台比较空闲的机器。
</div></div></div><div class="recent-post-item"><div class="post_cover left_radius"><a href="/2020/11/07/cadaa92fbbe2.html" title="Hadoop服役新服务器和推移旧服务器">     <img class="post_bg" src="https://oscimg.oschina.net/oscnet/up-aef8605c4e9e787f4e8913a79b770e9fa3e.png" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="Hadoop服役新服务器和推移旧服务器"></a></div><div class="recent-post-info"><a class="article-title" href="/2020/11/07/cadaa92fbbe2.html" title="Hadoop服役新服务器和推移旧服务器">Hadoop服役新服务器和推移旧服务器</a><div class="article-meta-wrap"><span class="post-meta-date"><i class="far fa-calendar-alt"></i><span class="article-meta-label">发表于</span><time datetime="2020-11-07T05:18:23.000Z" title="发表于 2020-11-07 13:18:23">2020-11-07</time></span><span class="article-meta"><span class="article-meta__separator">|</span><i class="fas fa-inbox"></i><a class="article-meta__categories" href="/categories/%E5%A4%A7%E6%95%B0%E6%8D%AE/">大数据</a></span><span class="article-meta tags"><span class="article-meta__separator">|</span><i class="fas fa-tag"></i><a class="article-meta__tags" href="/tags/Hadoop/">Hadoop</a></span></div><div class="content">服役新服务器1）需求随着公司业务的增长，数据量越来越大，原有的数据节点的容量已经不能满足存储数据的需求，需要在原有集群基础上动态添加新的数据节点。
2）环境准备新服务器安装Hadoop相关的环境，配置好hostname和hosts。拷贝/etc/profile.d/my_env.sh并执行source /etc/profile

注：通常采用镜像拷贝或者克隆的方式在新机器上克隆就机器的镜像。如果是使用旧镜像的话，克隆完成后记得删除Hadoop目录下的data和logs。

3）服役新节点具体步骤（1）直接启动DataNode，即可关联到集群
[lytfly@hadoop105 hadoop-3.1.4]$ hdfs --daemon start datanode[lytfly@hadoop105 hadoop-3.1.4]$ yarn --daemon start nodemanager
4）在白名单中增加新服役的服务器（1）在白名单whitelist和workers文件中增加hadoop104、hadoop105节点，并重启集群
[lytfly@hadoop102 hadoop]$  ...</div></div></div><div class="recent-post-item"><div class="post_cover right_radius"><a href="/2020/11/07/a8fda6be4c8d.html" title="HDFS—存储优化（纠删码）">     <img class="post_bg" src="https://oscimg.oschina.net/oscnet/up-87ba27811580330155a29254c76223dd59f.png" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="HDFS—存储优化（纠删码）"></a></div><div class="recent-post-info"><a class="article-title" href="/2020/11/07/a8fda6be4c8d.html" title="HDFS—存储优化（纠删码）">HDFS—存储优化（纠删码）</a><div class="article-meta-wrap"><span class="post-meta-date"><i class="far fa-calendar-alt"></i><span class="article-meta-label">发表于</span><time datetime="2020-11-07T05:17:25.000Z" title="发表于 2020-11-07 13:17:25">2020-11-07</time></span><span class="article-meta"><span class="article-meta__separator">|</span><i class="fas fa-inbox"></i><a class="article-meta__categories" href="/categories/%E5%A4%A7%E6%95%B0%E6%8D%AE/">大数据</a></span><span class="article-meta tags"><span class="article-meta__separator">|</span><i class="fas fa-tag"></i><a class="article-meta__tags" href="/tags/Hadoop/">Hadoop</a><span class="article-meta__link">•</span><a class="article-meta__tags" href="/tags/HDFS/">HDFS</a></span></div><div class="content">纠删码原理HDFS默认情况下，一个文件有3个副本，这样提高了数据的可靠性，但也带来了2倍的冗余开销。
Hadoop3.x引入了纠删码，采用计算的方式，可以节省约50％左右的存储空间。
此种方式节约了空间，但是会增加cpu的计算。
纠删码策略是给具体一个路径设置。所有往此路径下存储的文件，都会执行此策略。
默认只开启对RS-6-3-1024k策略的支持，如要使用别的策略需要提前启用。

1）纠删码操作相关的命令
2）查看当前支持的纠删码策略
3）纠删码策略解释RS-3-2-1024k***_：使用RS编码，每3个数据单元，生成2个校验单元，共5个单元，也就是说：这5个单元中，只要有任意的3个单元存在（不管是数据单元还是校验单元，只要总数=3），就可以得到原始数据。每个单元的大小是1024k=1024_1024=1048576。

RS-10-4-1024k***_：使用RS编码，每10个数据单元（cell），生成4个校验单元，共14个单元，也就是说：这14个单元中，只要有任意的10个单元存在（不管是数据单元还是校验单元，只要总数=10），就可以得到原始数据。每个单元的大小是1024k=1 ...</div></div></div><nav id="pagination"><div class="pagination"><a class="extend prev" rel="prev" href="/page/2/#content-inner"><i class="fas fa-chevron-left fa-fw"></i></a><a class="page-number" href="/">1</a><a class="page-number" href="/page/2/#content-inner">2</a><span class="page-number current">3</span><a class="page-number" href="/page/4/#content-inner">4</a><a class="page-number" href="/page/5/#content-inner">5</a><a class="extend next" rel="next" href="/page/4/#content-inner"><i class="fas fa-chevron-right fa-fw"></i></a></div></nav></div><div class="aside-content" id="aside-content"><div class="card-widget card-info"><div class="is-center"><div class="avatar-img"><img src="/img/zhiyao_avatar.jpg" onerror="this.onerror=null;this.src='/img/friend_404.gif'" alt="avatar"/></div><div class="author-info__name">liuyuantao</div><div class="author-info__description">一个有趣、有料、有内涵的地方!</div></div><div class="card-info-data"><div class="card-info-data-item is-center"><a href="/archives/"><div class="headline">文章</div><div class="length-num">41</div></a></div><div class="card-info-data-item is-center"><a href="/tags/"><div class="headline">标签</div><div class="length-num">14</div></a></div><div class="card-info-data-item is-center"><a href="/categories/"><div class="headline">分类</div><div class="length-num">4</div></a></div></div></div><div class="card-widget card-announcement"><div class="item-headline"><i class="fas fa-bullhorn card-announcement-animation"></i><span>公告</span></div><div class="announcement_content"><p><strong style="color:#858585;font-size:20px;">扫描二维码,关注我们</strong><br/><span style="color:#49b1f5;">了解更多全栈技能!</span></p><img src="/img/qrcode_for_gh_da91cebd00cc_258.jpg"></div></div><div class="sticky_layout"><div class="card-widget card-recent-post"><div class="item-headline"><i class="fas fa-history"></i><span>最新文章</span></div><div class="aside-list"><div class="aside-list-item"><a class="thumbnail" href="/2021/11/19/4e544b17c3ac.html" title="Nginx日志按天自动切割"><img src="https://cdn.pixabay.com/photo/2021/10/25/17/16/nature-6741602__340.jpg" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="Nginx日志按天自动切割"/></a><div class="content"><a class="title" href="/2021/11/19/4e544b17c3ac.html" title="Nginx日志按天自动切割">Nginx日志按天自动切割</a><time datetime="2021-11-19T09:40:06.000Z" title="发表于 2021-11-19 17:40:06">2021-11-19</time></div></div><div class="aside-list-item"><a class="thumbnail" href="/2021/11/18/b4485f4970ce.html" title="ClickHouse新特性之SQL化用户配置"><img src="https://cdn.pixabay.com/photo/2017/01/31/18/21/books-2026194__340.png" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="ClickHouse新特性之SQL化用户配置"/></a><div class="content"><a class="title" href="/2021/11/18/b4485f4970ce.html" title="ClickHouse新特性之SQL化用户配置">ClickHouse新特性之SQL化用户配置</a><time datetime="2021-11-18T05:55:38.000Z" title="发表于 2021-11-18 13:55:38">2021-11-18</time></div></div><div class="aside-list-item"><a class="thumbnail" href="/2021/11/18/6d7187a575ff.html" title="MergeTree引擎的固定/自适应索引粒度"><img src="https://oscimg.oschina.net/oscnet/up-8479e16ac3738d1e8243838c11c9e96aba9.png" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="MergeTree引擎的固定/自适应索引粒度"/></a><div class="content"><a class="title" href="/2021/11/18/6d7187a575ff.html" title="MergeTree引擎的固定/自适应索引粒度">MergeTree引擎的固定/自适应索引粒度</a><time datetime="2021-11-18T05:52:18.000Z" title="发表于 2021-11-18 13:52:18">2021-11-18</time></div></div><div class="aside-list-item"><a class="thumbnail" href="/2021/11/18/23f3f36da036.html" title="无缝更改ClickHouse物化视图SELECT逻辑的方法"><img src="https://oscimg.oschina.net/oscnet/up-ac406843f42bfac15c16815aeb376bca9ad.png" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="无缝更改ClickHouse物化视图SELECT逻辑的方法"/></a><div class="content"><a class="title" href="/2021/11/18/23f3f36da036.html" title="无缝更改ClickHouse物化视图SELECT逻辑的方法">无缝更改ClickHouse物化视图SELECT逻辑的方法</a><time datetime="2021-11-18T05:43:50.000Z" title="发表于 2021-11-18 13:43:50">2021-11-18</time></div></div><div class="aside-list-item"><a class="thumbnail" href="/2021/11/18/fec2d399fcb4.html" title="物化视图简介与ClickHouse中的应用示例"><img src="https://upload-images.jianshu.io/upload_images/195230-462138b5b8e6e815.png" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="物化视图简介与ClickHouse中的应用示例"/></a><div class="content"><a class="title" href="/2021/11/18/fec2d399fcb4.html" title="物化视图简介与ClickHouse中的应用示例">物化视图简介与ClickHouse中的应用示例</a><time datetime="2021-11-18T05:31:13.000Z" title="发表于 2021-11-18 13:31:13">2021-11-18</time></div></div></div></div><div class="card-widget card-categories"><div class="item-headline">
            <i class="fas fa-folder-open"></i>
            <span>分类</span>
            
            </div>
            <ul class="card-category-list" id="aside-cat-list">
            <li class="card-category-list-item "><a class="card-category-list-link" href="/categories/Spring-SpringBoot/"><span class="card-category-list-name">Spring&SpringBoot</span><span class="card-category-list-count">2</span></a></li><li class="card-category-list-item "><a class="card-category-list-link" href="/categories/%E4%B8%AD%E9%97%B4%E4%BB%B6/"><span class="card-category-list-name">中间件</span><span class="card-category-list-count">2</span></a></li><li class="card-category-list-item "><a class="card-category-list-link" href="/categories/%E5%A4%A7%E6%95%B0%E6%8D%AE/"><span class="card-category-list-name">大数据</span><span class="card-category-list-count">34</span></a></li><li class="card-category-list-item "><a class="card-category-list-link" href="/categories/%E8%BF%90%E7%BB%B4/"><span class="card-category-list-name">运维</span><span class="card-category-list-count">1</span></a></li>
            </ul></div><div class="card-widget card-tags"><div class="item-headline"><i class="fas fa-tags"></i><span>标签</span></div><div class="card-tag-cloud"><a href="/tags/Hadoop/" style="font-size: 1.45em; color: rgb(29, 18, 195)">Hadoop</a><a href="/tags/HDFS/" style="font-size: 1.39em; color: rgb(67, 37, 146)">HDFS</a><a href="/tags/ClickHouse/" style="font-size: 1.27em; color: rgb(1, 73, 96)">ClickHouse</a><a href="/tags/Yarn/" style="font-size: 1.33em; color: rgb(150, 127, 36)">Yarn</a><a href="/tags/%E8%84%9A%E6%9C%AC/" style="font-size: 1.15em; color: rgb(88, 131, 138)">脚本</a><a href="/tags/Nginx/" style="font-size: 1.15em; color: rgb(178, 37, 116)">Nginx</a><a href="/tags/%E8%BF%90%E7%BB%B4/" style="font-size: 1.15em; color: rgb(128, 60, 172)">运维</a><a href="/tags/HBase/" style="font-size: 1.15em; color: rgb(0, 104, 140)">HBase</a><a href="/tags/%E4%B8%AD%E9%97%B4%E4%BB%B6/" style="font-size: 1.21em; color: rgb(40, 97, 23)">中间件</a><a href="/tags/Zookeeper/" style="font-size: 1.15em; color: rgb(40, 178, 154)">Zookeeper</a><a href="/tags/Hive/" style="font-size: 1.27em; color: rgb(147, 104, 22)">Hive</a><a href="/tags/Kafka/" style="font-size: 1.15em; color: rgb(176, 132, 95)">Kafka</a><a href="/tags/MapReduce/" style="font-size: 1.15em; color: rgb(157, 129, 39)">MapReduce</a><a href="/tags/Flume/" style="font-size: 1.15em; color: rgb(72, 148, 10)">Flume</a></div></div><div class="card-widget card-archives"><div class="item-headline"><i class="fas fa-archive"></i><span>归档</span></div><ul class="card-archive-list"><li class="card-archive-list-item"><a class="card-archive-list-link" href="/archives/2021/11/"><span class="card-archive-list-date">十一月 2021</span><span class="card-archive-list-count">8</span></a></li><li class="card-archive-list-item"><a class="card-archive-list-link" href="/archives/2021/10/"><span class="card-archive-list-date">十月 2021</span><span class="card-archive-list-count">1</span></a></li><li class="card-archive-list-item"><a class="card-archive-list-link" href="/archives/2021/09/"><span class="card-archive-list-date">九月 2021</span><span class="card-archive-list-count">1</span></a></li><li class="card-archive-list-item"><a class="card-archive-list-link" href="/archives/2021/07/"><span class="card-archive-list-date">七月 2021</span><span class="card-archive-list-count">1</span></a></li><li class="card-archive-list-item"><a class="card-archive-list-link" href="/archives/2020/11/"><span class="card-archive-list-date">十一月 2020</span><span class="card-archive-list-count">27</span></a></li><li class="card-archive-list-item"><a class="card-archive-list-link" href="/archives/2020/09/"><span class="card-archive-list-date">九月 2020</span><span class="card-archive-list-count">2</span></a></li><li class="card-archive-list-item"><a class="card-archive-list-link" href="/archives/2019/11/"><span class="card-archive-list-date">十一月 2019</span><span class="card-archive-list-count">1</span></a></li></ul></div><div class="card-widget card-webinfo"><div class="item-headline"><i class="fas fa-chart-line"></i><span>网站资讯</span></div><div class="webinfo"><div class="webinfo-item"><div class="item-name">文章数目 :</div><div class="item-count">41</div></div><div class="webinfo-item"><div class="item-name">已运行时间 :</div><div class="item-count" id="runtimeshow" data-publishDate="2020-08-31T16:00:00.000Z"></div></div><div class="webinfo-item"><div class="item-name">本站总字数 :</div><div class="item-count">59.5k</div></div><div class="webinfo-item"><div class="item-name">本站访客数 :</div><div class="item-count" id="busuanzi_value_site_uv"></div></div><div class="webinfo-item"><div class="item-name">本站总访问量 :</div><div class="item-count" id="busuanzi_value_site_pv"></div></div><div class="webinfo-item"><div class="item-name">最后更新时间 :</div><div class="item-count" id="last-push-date" data-lastPushDate="2021-11-23T10:55:57.650Z"></div></div></div></div></div></div></main><footer id="footer"><div id="footer-wrap"><div class="copyright">&copy;2020 - 2021 By liuyuantao</div><div class="framework-info"><span>框架 </span><a target="_blank" rel="noopener external nofollow noreferrer" href="https://hexo.io">Hexo</a><span class="footer-separator">|</span><span>主题 </span><a target="_blank" rel="noopener external nofollow noreferrer" href="https://github.com/jerryc127/hexo-theme-butterfly">Butterfly</a></div><div class="footer_custom_text">扫描二维码,关注我们!了解更多全栈技能!<br/><img style="width:100px;height:100px;" src="/img/qrcode_for_gh_da91cebd00cc_258.jpg" ></div></div></footer></div><div id="rightside"><div id="rightside-config-hide"><button id="font-plus" type="button" title="放大字体"><i class="fas fa-plus"></i></button><button id="font-minus" type="button" title="缩小字体"><i class="fas fa-minus"></i></button><button id="translateLink" type="button" title="简繁转换">简</button><button id="darkmode" type="button" title="浅色和深色模式转换"><i class="fas fa-adjust"></i></button><button id="hide-aside-btn" type="button" title="单栏和双栏切换"><i class="fas fa-arrows-alt-h"></i></button></div><div id="rightside-config-show"><button id="rightside_config" type="button" title="设置"><i class="fas fa-cog fa-spin"></i></button><button id="go-up" type="button" title="回到顶部"><i class="fas fa-arrow-up"></i></button></div></div><div id="local-search"><div class="search-dialog"><div class="search-dialog__title" id="local-search-title">本地搜索</div><div id="local-input-panel"><div id="local-search-input"><div class="local-search-box"><input class="local-search-box--input" placeholder="搜索文章" type="text"/></div></div></div><hr/><div id="local-search-results"></div><span class="search-close-button"><i class="fas fa-times"></i></span></div><div id="search-mask"></div></div><div><script src="/js/utils.js"></script><script src="/js/main.js"></script><script src="/js/tw_cn.js"></script><script src="https://cdn.jsdelivr.net/npm/medium-zoom/dist/medium-zoom.min.js"></script><script src="https://cdn.jsdelivr.net/npm/instant.page/instantpage.min.js" type="module"></script><script src="https://cdn.jsdelivr.net/npm/node-snackbar/dist/snackbar.min.js"></script><script src="/js/search/local-search.js"></script><script>var preloader = {
  endLoading: () => {
    document.body.style.overflow = 'auto';
    document.getElementById('loading-box').classList.add("loaded")
  },
  initLoading: () => {
    document.body.style.overflow = '';
    document.getElementById('loading-box').classList.remove("loaded")

  }
}
window.addEventListener('load',preloader.endLoading())</script><div class="js-pjax"></div><script defer="defer" id="fluttering_ribbon" mobile="false" src="https://cdn.jsdelivr.net/npm/butterfly-extsrc@1/dist/canvas-fluttering-ribbon.min.js"></script><script async data-pjax src="//busuanzi.ibruce.info/busuanzi/2.3/busuanzi.pure.mini.js"></script></div></body></html>